import asyncio
import time
import random

import aiofiles
import aiohttp
from lxml import etree

headers = {
    "user-agent": "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Mobile Safari/537.36 Edg/131.0.0.0"
}
# proxies = "http://114.231.82.246:8089"

async def request_url_list():
    """ 获取所有章节url """
    base_url = "https://guoxue.httpcn.com/book/sgyy/"

    async with aiohttp.ClientSession() as session:
        async with session.get(base_url, headers=headers) as resp:
            html = etree.HTML(await resp.text())
            # 定位到每个标题a标签
            a_list = html.xpath("/html/body/div[4]/div[2]/div[1]/div[1]/div[2]/div[3]/div/div/ul/li/span/a")
            # 获取所有章节的url地址列表
            urls = []
            for a in a_list:
                json = {
                    "url" : a.xpath("@href")[0],
                    "title" : a.xpath(".//text()")[0]
                }
                urls.append(json)
            return urls


async def write_files(text,title):
    """ 提取响应内容写入文件 """
    try:
        res_html = etree.HTML(text)
        # 提取正文
        p_list = res_html.xpath("/html/body/div[4]/div[2]/div[1]/div/div[1]/div[3]/p")
        content = ''
        for p in p_list:
            content += p.xpath(".//text()")[0] + "/n"
        async with aiofiles.open("三国演义/" + title + ".text", mode="w", encoding="utf-8") as f:
            await f.write(content)
    except Exception as e:
        print(f"error:{e}")


async def aio_download(url_r,title):
    """ 异步请求获取每章内容 """
    try:
        async with aiohttp.ClientSession() as session:
            # time.sleep(random.uniform(1,5))
            await asyncio.sleep(random.uniform(2, 5))
            async with session.get("https:" + url_r, headers=headers) as res:
                # 确保响应状态码是200（OK）
                if res.status == 200:
                    await write_files(await res.text(), title)
                else:
                    print(f"Failed to download file: Status code {res.status}")
    except Exception as e:
        print(f"error:{e}")


async def main():
    """ 设置异步任务 """
    urls = await request_url_list()
    tasks = []
    for url_r in urls:
        # print(url_r["url"],url_r["title"])
        tasks.append(asyncio.create_task(aio_download(url_r["url"],url_r["title"])))
    await asyncio.wait(tasks)


if __name__ == '__main__':
    asyncio.run(main())
